{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "e963fd90",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "输入张量形状: torch.Size([2, 8, 64])\n",
      "旋转矩阵形状: torch.Size([8, 64])\n",
      "输出张量形状: torch.Size([2, 8, 64])\n"
     ]
    }
   ],
   "source": [
    "import torch\n",
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def get_rotary_matrix(dim: int, seq_len: int, base: int = 10000):\n",
    "    \"\"\"\n",
    "    生成旋转位置编码矩阵\n",
    "    Args:\n",
    "        dim: 编码维度\n",
    "        seq_len: 序列长度\n",
    "        base: 缩放基数\n",
    "    \"\"\"\n",
    "    # 生成位置索引向量 [seq_len]\n",
    "    position = torch.arange(seq_len).unsqueeze(1)  # [seq_len, 1]\n",
    "    \n",
    "    # 生成维度索引向量 [dim/2]\n",
    "    div_term = torch.exp(torch.arange(0, dim, 2) * (-np.log(base) / dim))\n",
    "    \n",
    "    # 计算旋转角度矩阵 [seq_len, dim/2]\n",
    "    theta = position * div_term\n",
    "    \n",
    "    # 构建旋转矩阵 [seq_len, dim]\n",
    "    rotary_matrix = torch.zeros(seq_len, dim)\n",
    "    rotary_matrix[:, 0::2] = torch.cos(theta)  # 偶数位置存放cos值\n",
    "    rotary_matrix[:, 1::2] = torch.sin(theta)  # 奇数位置存放sin值\n",
    "    \n",
    "    return rotary_matrix\n",
    "\n",
    "def apply_rotary_embedding(x: torch.Tensor, rotary_matrix: torch.Tensor):\n",
    "    \"\"\"\n",
    "    应用旋转位置编码\n",
    "    Args:\n",
    "        x: 输入张量 [batch_size, seq_len, dim]\n",
    "        rotary_matrix: 旋转矩阵 [seq_len, dim]\n",
    "    \"\"\"\n",
    "    # 将输入分割为实部和虚部\n",
    "    x_real = x[..., 0::2]  # 偶数维度\n",
    "    x_imag = x[..., 1::2]  # 奇数维度\n",
    "    \n",
    "    # 获取旋转矩阵的cos和sin部分\n",
    "    cos = rotary_matrix[:, 0::2]  # [seq_len, dim/2]\n",
    "    sin = rotary_matrix[:, 1::2]  # [seq_len, dim/2]\n",
    "    \n",
    "    # 应用复数旋转变换\n",
    "    # 实部: x_real * cos - x_imag * sin\n",
    "    # 虚部: x_real * sin + x_imag * cos\n",
    "    out_real = x_real * cos - x_imag * sin\n",
    "    out_imag = x_real * sin + x_imag * cos\n",
    "    \n",
    "    # 交错合并实部和虚部\n",
    "    out = torch.zeros_like(x)\n",
    "    out[..., 0::2] = out_real\n",
    "    out[..., 1::2] = out_imag\n",
    "    \n",
    "    return out\n",
    "\n",
    "# 测试代码\n",
    "if __name__ == \"__main__\":\n",
    "    # 参数设置\n",
    "    batch_size = 2\n",
    "    seq_len = 8\n",
    "    dim = 64\n",
    "    \n",
    "    # 生成随机输入\n",
    "    x = torch.randn(batch_size, seq_len, dim)\n",
    "    print(f\"输入张量形状: {x.shape}\")\n",
    "    \n",
    "    # 生成旋转矩阵\n",
    "    rotary_matrix = get_rotary_matrix(dim, seq_len)\n",
    "    print(f\"旋转矩阵形状: {rotary_matrix.shape}\")\n",
    "\n",
    "    # 应用旋转位置编码\n",
    "    output = apply_rotary_embedding(x, rotary_matrix)\n",
    "    print(f\"输出张量形状: {output.shape}\")\n"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 3,
   "id": "c364ea02",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "相对位置编码矩阵形状: (10, 10, 8)\n",
      "\n",
      "位置(0,1)的编码向量: [-0.84147098  0.54030231 -0.09983342  0.99500417 -0.00999983  0.99995\n",
      " -0.001       0.9999995 ]\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import matplotlib.pyplot as plt\n",
    "\n",
    "def get_relative_position_encoding(seq_len, d_model):\n",
    "    \"\"\"\n",
    "    # 生成相对位置编码矩阵\n",
    "    seq_len: 序列长度\n",
    "    d_model: 编码维度\n",
    "    \"\"\"\n",
    "    # 创建位置矩阵 (seq_len x seq_len)\n",
    "    positions = np.arange(seq_len)[:, np.newaxis] - np.arange(seq_len)[np.newaxis, :]\n",
    "    \n",
    "    # 生成不同频率的正弦波\n",
    "    div_term = np.exp(np.arange(0, d_model, 2) * -(np.log(10000.0) / d_model))\n",
    "    \n",
    "    # 初始化编码矩阵\n",
    "    pe = np.zeros((seq_len, seq_len, d_model))\n",
    "    \n",
    "    # 计算相对位置编码\n",
    "    # 使用正弦和余弦函数交替编码\n",
    "    pe[:, :, 0::2] = np.sin(positions[:, :, np.newaxis] * div_term[np.newaxis, np.newaxis, :])\n",
    "    pe[:, :, 1::2] = np.cos(positions[:, :, np.newaxis] * div_term[np.newaxis, np.newaxis, :])\n",
    "    \n",
    "    return pe\n",
    "\n",
    "# 测试参数\n",
    "seq_length = 10  # 序列长度\n",
    "d_model = 8      # 编码维度\n",
    "\n",
    "# 生成相对位置编码\n",
    "rel_pos_encoding = get_relative_position_encoding(seq_length, d_model)\n",
    "\n",
    "# 打印部分编码值\n",
    "print(\"相对位置编码矩阵形状:\", rel_pos_encoding.shape)\n",
    "print(\"\\n位置(0,1)的编码向量:\", rel_pos_encoding[0, 1])\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "c25f7ed9",
   "metadata": {},
   "source": [
    "# 全量训练、全量微调与参数高效微调解析\n",
    "\n",
    "## 1. 全量训练（Full Training）\n",
    "### 定义\n",
    "从零开始训练模型，**所有参数**随机初始化并通过目标数据集进行训练。\n",
    "\n",
    "### 核心特点\n",
    "- **数据需求**：需大规模标注数据  \n",
    "- **计算成本**：极高（所有参数参与梯度计算与更新）  \n",
    "- **适用场景**：  \n",
    "  - 无相关预训练模型时  \n",
    "  - 目标任务与预训练任务差异极大  \n",
    "\n",
    "### 训练流程\n",
    "```python\n",
    "model = Transformer()  # 随机初始化所有参数\n",
    "optimizer = Adam(model.parameters(), lr=5e-5)  # 优化全部参数\n",
    "for batch in dataset:\n",
    "    loss = model(batch).loss\n",
    "    loss.backward()\n",
    "    optimizer.step()\n",
    "\n"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "b9a9159f",
   "metadata": {},
   "source": [
    "## 全量微调 (Full Fine-tuning)\n",
    "\n",
    "### 基本概念\n",
    "全量微调是指在预训练语言模型（如BERT、GPT等）的基础上，**对所有参数**进行端到端的微调以适应特定下游任务。\n",
    "\n",
    "### 核心特点\n",
    "- **参数更新范围**：100%模型参数可训练\n",
    "- **典型学习率**：1e-5 ~ 5e-5（比预训练时小1-2个数量级）\n",
    "- **数据需求**：通常需要10,000+标注样本"
   ]
  },
  {
   "cell_type": "markdown",
   "id": "9e403702",
   "metadata": {},
   "source": [
    "# 参数高效微调（PEFT）技术详解\n",
    "\n",
    "## 基本概念\n",
    "参数高效微调（Parameter-Efficient Fine-tuning, PEFT）是一类通过**仅微调少量参数**来适配下游任务的技术，通常只更新不到5%的模型参数。\n",
    "\n",
    "## 核心优势\n",
    "- 🚀 **计算效率**：减少70-90%的训练资源消耗\n",
    "- 💾 **存储效率**：单个任务适配器仅需1-10MB存储\n",
    "- 🔄 **多任务支持**：通过切换不同适配模块实现单模型多任务\n",
    "\n",
    "## 主流方法\n",
    "\n",
    "### 1. LoRA（低秩适应）\n",
    "#### 工作原理\n",
    "```math\n",
    "\\Delta W = BA \\quad \\text{其中} \\quad B \\in \\mathbb{R}^{d×r}, A \\in \\mathbb{R}^{r×k}, r \\ll d"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.3"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
